{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'channels_last'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras.layers.core import Dense, Flatten\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.pooling import MaxPooling2D\n",
    "\n",
    "from keras import backend as K\n",
    "\n",
    "K.image_data_format()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "reshape原数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the model\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "    x_image = tf.reshape(x, [-1, 28, 28, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "卷积"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 25, 25, 36)\n",
      "(?, 12, 12, 36)\n",
      "(?, 9, 9, 81)\n",
      "(?, 4, 4, 81)\n"
     ]
    }
   ],
   "source": [
    "net = Conv2D(36, kernel_size = [4, 4], strides=[1, 1], activation='relu', padding='VALID', input_shape=[28, 28, 1])(x_image)\n",
    "print(net.shape)\n",
    "net = MaxPooling2D(pool_size = [2, 2])(net)\n",
    "print(net.shape)\n",
    "net = Conv2D(81, kernel_size = [4, 4], strides=[1, 1], activation='relu', padding='VALID')(net)\n",
    "print(net.shape)\n",
    "net = MaxPooling2D(pool_size = [2, 2])(net)\n",
    "print(net.shape)\n",
    "net = Flatten()(net)\n",
    "net = Dense(1024, activation='relu')(net)\n",
    "net = Dense(10, activation='softmax')(net)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "from keras.objectives import categorical_crossentropy\n",
    "cross_entropy = tf.reduce_mean(categorical_crossentropy(y_, net))\n",
    "l2_loss  = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)]) \n",
    "total_loss = cross_entropy + 4e-5 * l2_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_step = tf.train.RMSPropOptimizer(1e-3).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。\n",
    "然后我们运行3k个step(5 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100,entropy loss: 1.099778, l2_loss: 611.097534, total_loss: 1.124222\n",
      "0.72\n",
      "step 200,entropy loss: 0.219387, l2_loss: 632.344360, total_loss: 0.244680\n",
      "0.91\n",
      "step 300,entropy loss: 0.142848, l2_loss: 613.943909, total_loss: 0.167406\n",
      "1.0\n",
      "step 400,entropy loss: 0.154937, l2_loss: 602.667664, total_loss: 0.179044\n",
      "0.99\n",
      "step 500,entropy loss: 0.118495, l2_loss: 597.336548, total_loss: 0.142388\n",
      "0.99\n",
      "step 600,entropy loss: 0.036051, l2_loss: 589.927917, total_loss: 0.059648\n",
      "1.0\n",
      "step 700,entropy loss: 0.054399, l2_loss: 581.392944, total_loss: 0.077655\n",
      "1.0\n",
      "step 800,entropy loss: 0.010629, l2_loss: 570.775757, total_loss: 0.033460\n",
      "1.0\n",
      "step 900,entropy loss: 0.019835, l2_loss: 558.873169, total_loss: 0.042190\n",
      "1.0\n",
      "step 1000,entropy loss: 0.007387, l2_loss: 547.184082, total_loss: 0.029275\n",
      "1.0\n",
      "0.9879\n",
      "step 1100,entropy loss: 0.034718, l2_loss: 532.167786, total_loss: 0.056005\n",
      "1.0\n",
      "step 1200,entropy loss: 0.001115, l2_loss: 520.461609, total_loss: 0.021933\n",
      "1.0\n",
      "step 1300,entropy loss: 0.037112, l2_loss: 507.681396, total_loss: 0.057420\n",
      "1.0\n",
      "step 1400,entropy loss: 0.039483, l2_loss: 497.640594, total_loss: 0.059388\n",
      "0.99\n",
      "step 1500,entropy loss: 0.106476, l2_loss: 487.181702, total_loss: 0.125963\n",
      "1.0\n",
      "step 1600,entropy loss: 0.010969, l2_loss: 474.190704, total_loss: 0.029937\n",
      "1.0\n",
      "step 1700,entropy loss: 0.037530, l2_loss: 463.995880, total_loss: 0.056090\n",
      "1.0\n",
      "step 1800,entropy loss: 0.029077, l2_loss: 456.504639, total_loss: 0.047337\n",
      "1.0\n",
      "step 1900,entropy loss: 0.005958, l2_loss: 449.697113, total_loss: 0.023946\n",
      "1.0\n",
      "step 2000,entropy loss: 0.003875, l2_loss: 442.050415, total_loss: 0.021557\n",
      "1.0\n",
      "0.9923\n",
      "step 2100,entropy loss: 0.018738, l2_loss: 436.340240, total_loss: 0.036192\n",
      "1.0\n",
      "step 2200,entropy loss: 0.047189, l2_loss: 433.209991, total_loss: 0.064517\n",
      "1.0\n",
      "step 2300,entropy loss: 0.020682, l2_loss: 421.840668, total_loss: 0.037556\n",
      "1.0\n",
      "step 2400,entropy loss: 0.011767, l2_loss: 412.652954, total_loss: 0.028273\n",
      "1.0\n",
      "step 2500,entropy loss: 0.021746, l2_loss: 408.110931, total_loss: 0.038070\n",
      "1.0\n",
      "step 2600,entropy loss: 0.033477, l2_loss: 406.830414, total_loss: 0.049750\n",
      "1.0\n",
      "step 2700,entropy loss: 0.020972, l2_loss: 404.491272, total_loss: 0.037152\n",
      "1.0\n",
      "step 2800,entropy loss: 0.008924, l2_loss: 397.378021, total_loss: 0.024819\n",
      "1.0\n",
      "step 2900,entropy loss: 0.001174, l2_loss: 385.014984, total_loss: 0.016575\n",
      "1.0\n",
      "step 3000,entropy loss: 0.017414, l2_loss: 381.244263, total_loss: 0.032664\n",
      "1.0\n",
      "0.9912\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "for step in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    lr = 0.01\n",
    "    _, loss, l2_loss_value, total_loss_value = sess.run([train_step, cross_entropy, l2_loss, total_loss],\n",
    "                                                      feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr})\n",
    "    if (step+1) % 100 == 0:\n",
    "        print('step %d,entropy loss: %f, l2_loss: %f, total_loss: %f' % (step+1, loss, l2_loss_value, total_loss_value))\n",
    "        correct_prediction = tf.equal(tf.argmax(net, 1), tf.argmax(y_, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "    if (step+1) % 1000 == 0:\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证我们模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 卷积\n",
    "- 池化\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 卷积kernel size\n",
    "  - 卷积kernel 数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
